package pl.pelcra.cesar.spider.webcrawler;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.zip.GZIPInputStream;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParamBean;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import pl.pelcra.cesar.spider.webcrawler.interfaces.HttpRequestMethod;
import edu.uci.ics.crawler4j.crawler.Configurable;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.fetcher.CustomFetchStatus;
import edu.uci.ics.crawler4j.fetcher.PageFetchResult;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * Class fetch page from server and returns result as PageFetchResult from
 * function fetchHeader
 * 
 * @author margielewski
 * 
 */
public class PageFetcher extends Configurable {

	private static class GzipDecompressingEntity extends HttpEntityWrapper {

		public GzipDecompressingEntity(final HttpEntity entity) {
			super(entity);
		}

		@Override
		public InputStream getContent() throws IOException,
				IllegalStateException {

			// the wrapped entity's getContent() decides about repeatability
			InputStream wrappedin = this.wrappedEntity.getContent();

			return new GZIPInputStream(wrappedin);
		}

		@Override
		public long getContentLength() {
			// length of ungzipped content is not known
			return -1;
		}

	}

	private static final Logger logger = Logger.getLogger(PageFetcher.class);

	private ThreadSafeClientConnManager connectionManager;

	private DefaultHttpClient httpClient;

	private final long lastFetchTime = 0;

	private final Object mutex = new Object();

	public PageFetcher(CrawlConfig config) {
		super(config);

		HttpParams params = new BasicHttpParams();
		HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
		paramsBean.setVersion(HttpVersion.HTTP_1_1);
		paramsBean.setContentCharset("UTF-8");
		paramsBean.setUseExpectContinue(false);

		params.setParameter(CoreProtocolPNames.USER_AGENT,
				config.getUserAgentString());
		params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,
				config.getSocketTimeout());
		params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
				config.getConnectionTimeout());

		params.setBooleanParameter("http.protocol.handle-redirects", false);

		SchemeRegistry schemeRegistry = new SchemeRegistry();
		schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory
				.getSocketFactory()));

		if (config.isIncludeHttpsPages()) {
			schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory
					.getSocketFactory()));
		}

		this.connectionManager = new ThreadSafeClientConnManager(schemeRegistry);
		this.connectionManager.setMaxTotal(config.getMaxTotalConnections());
		this.connectionManager.setDefaultMaxPerRoute(config
				.getMaxConnectionsPerHost());
		logger.setLevel(Level.INFO);
		this.httpClient = new DefaultHttpClient(this.connectionManager, params);

		if (config.getProxyHost() != null && !config.getProxyHost().isEmpty()) {

			if (config.getProxyUsername() != null
					&& !config.getProxyUsername().isEmpty()) {
				this.httpClient.getCredentialsProvider()
						.setCredentials(
								new AuthScope(config.getProxyHost(),
										config.getProxyPort()),
								new UsernamePasswordCredentials(config
										.getProxyUsername(), config
										.getProxyPassword()));
			}

			HttpHost proxy = new HttpHost(config.getProxyHost(),
					config.getProxyPort());
			this.httpClient.getParams().setParameter(
					ConnRoutePNames.DEFAULT_PROXY, proxy);
		}

		this.httpClient.addResponseInterceptor(new HttpResponseInterceptor() {

			@Override
			public void process(final HttpResponse response,
					final HttpContext context) throws HttpException,
					IOException {
				HttpEntity entity = response.getEntity();
				Header contentEncoding = entity.getContentEncoding();
				if (contentEncoding != null) {
					HeaderElement[] codecs = contentEncoding.getElements();
					for (HeaderElement codec : codecs) {
						if (codec.getName().equalsIgnoreCase("gzip")) {
							response.setEntity(new GzipDecompressingEntity(
									response.getEntity()));
							return;
						}
					}
				}
			}

		});
	}

	/**
	 * function fetch page from webUrl object and return PageFetchResult
	 * 
	 * @param webUrl
	 *            webUrl object with link to get page
	 * @param httpRequestMethod
	 *            enum with two available options POST or GET
	 * @param parameterValuePairs
	 *            parameters to POST method
	 * @param isCookieRequired
	 *            boolean value tells that cookies are required
	 * @return return PageFetchResult object for current url
	 */
	public PageFetchResult fetchHeader(WebURL webUrl,
			HttpRequestMethod httpRequestMethod,
			List<BasicNameValuePair> parameterValuePairs,
			boolean isCookieRequired) {
		PageFetchResult fetchResult = new PageFetchResult();
		String toFetchURL = webUrl.getURL();
		HttpGet httpGet = null;
		HttpPost httpPost = null;
		try {
			HttpResponse response = null;

			if (httpRequestMethod == HttpRequestMethod.GET) {

				if (parameterValuePairs != null) {
					StringBuilder sb = new StringBuilder();
					for (int i = 0; i < parameterValuePairs.size(); i++) {
						if (i == 0) {
							sb.append("?");
						}
						sb.append(String.format("%s=%s&", parameterValuePairs
								.get(i).getName(), parameterValuePairs.get(i)
								.getValue()));
					}

					if (sb.length() > 0) {
						if (toFetchURL.endsWith("/")) {
							toFetchURL = toFetchURL.substring(0,
									toFetchURL.length() - 2);
						}
						toFetchURL += sb.toString();
						if (toFetchURL.endsWith("&")) {
							toFetchURL = toFetchURL.substring(0,
									toFetchURL.length() - 2);
						}
					}
				}
				httpGet = new HttpGet(toFetchURL);
				httpGet.addHeader("Accept-Encoding", "gzip");
				response = this.httpClient.execute(httpGet);
			} else if (httpRequestMethod == HttpRequestMethod.POST) {
				httpPost = new HttpPost(toFetchURL);

				if (isCookieRequired) {
					CookieStore cookieStore = new BasicCookieStore();
					HttpContext localContext = new BasicHttpContext();
					localContext.setAttribute(ClientContext.COOKIE_STORE,
							cookieStore);

					this.httpClient.execute(httpGet, localContext);

					for (Cookie cookie : cookieStore.getCookies()) {
						String cook = String.format("%s=%s; path=%s",
								cookie.getName(), cookie.getValue(), "/");
						httpPost.setHeader("Cookie", cook);
					}
				}

				if (parameterValuePairs != null) {
					UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(
							parameterValuePairs);
					urlEncodedFormEntity.setContentEncoding(HTTP.UTF_8);
					httpPost.setEntity(urlEncodedFormEntity);
				}

				httpPost.addHeader("Accept-Encoding", "gzip");
				response = this.httpClient.execute(httpPost);
			}

			fetchResult.setEntity(response.getEntity());

			int statusCode = response.getStatusLine().getStatusCode();
			if (statusCode != HttpStatus.SC_OK) {
				if (statusCode != HttpStatus.SC_NOT_FOUND) {
					if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY
							|| statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
						Header header = response.getFirstHeader("Location");
						if (header != null) {
							String movedToUrl = header.getValue();
							movedToUrl = URLCanonicalizer.getCanonicalURL(
									movedToUrl, toFetchURL);
							fetchResult.setMovedToUrl(movedToUrl);
						}
						fetchResult.setStatusCode(statusCode);
						return fetchResult;
					}
					logger.info("Failed: "
							+ response.getStatusLine().toString()
							+ ", while fetching " + toFetchURL);
				}
				fetchResult.setStatusCode(response.getStatusLine()
						.getStatusCode());
				return fetchResult;
			}

			fetchResult.setFetchedUrl(toFetchURL);
			String uri = null;

			if (httpRequestMethod == HttpRequestMethod.GET) {
				uri = httpGet.getURI().toString();
			} else if (httpRequestMethod == HttpRequestMethod.POST) {
				uri = httpPost.getURI().toString();
			}

			if (!uri.equals(toFetchURL)) {
				if (!URLCanonicalizer.getCanonicalURL(uri).equals(toFetchURL)) {
					fetchResult.setFetchedUrl(uri);
				}
			}

			if (fetchResult.getEntity() != null) {
				long size = fetchResult.getEntity().getContentLength();
				if (size == -1) {
					Header length = response.getLastHeader("Content-Length");
					if (length == null) {
						length = response.getLastHeader("Content-length");
					}
					if (length != null) {
						size = Integer.parseInt(length.getValue());
					} else {
						size = -1;
					}
				}
				if (size > this.config.getMaxDownloadSize()) {
					fetchResult.setStatusCode(CustomFetchStatus.PageTooBig);
					return fetchResult;
				}

				fetchResult.setStatusCode(HttpStatus.SC_OK);
				return fetchResult;

			} else {
				if (httpGet != null) {
					httpGet.abort();
				}
				if (httpPost != null) {
					httpPost.abort();
				}
			}
		} catch (IOException e) {
			logger.error("Fatal transport error: " + e.getMessage()
					+ " while fetching " + toFetchURL + " (link found in doc #"
					+ webUrl.getParentDocid() + ")");
			fetchResult.setStatusCode(CustomFetchStatus.FatalTransportError);
			return fetchResult;
		} catch (IllegalStateException e) {
			// ignoring exceptions that occur because of not registering https
			// and other schemes
		} catch (Exception e) {
			if (e.getMessage() == null) {
				logger.error("Error while fetching " + webUrl.getURL());
			} else {
				logger.error(e.getMessage() + " while fetching "
						+ webUrl.getURL());
			}
		} finally {
			try {
				if (fetchResult.getEntity() == null) {
					if (httpGet != null) {
						httpGet.abort();
					} else if (httpPost != null) {
						httpPost.abort();
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		fetchResult.setStatusCode(CustomFetchStatus.UnknownError);
		return fetchResult;
	}
}
